import base64
import datetime
from google.appengine.ext import db
from google.appengine.api import memcache

class TagIndex(db.Model):
  word = db.StringProperty()
  tags = db.StringListProperty()
  
class Tag(db.Model):
  name = db.StringProperty()
  records = db.ListProperty(db.Key)
  count = db.IntegerProperty(default = 0)
  related = db.StringListProperty()
  
class Record(db.Model):
  date = db.DateTimeProperty()
  dfmt = db.StringProperty()
  value = db.FloatProperty()
  unit = db.StringProperty()
  tags = db.StringListProperty()
  source = db.StringProperty()
  inputtime = db.DateTimeProperty(default = datetime.datetime.now())

class History(db.Model):
  tags = db.StringProperty()
  qcount = db.IntegerProperty(default = 1)
  found = db.BooleanProperty(default = True)
  timestamp = db.DateTimeProperty(default = datetime.datetime.now())

def GenRecordKeyName(date, tags):
  tags.sort()
  return 'r_'+base64.b64encode(" ".join(tags).encode("utf-8")+date.strftime("%Y%m%d%H%M%S"))

def GenTagKeyName(tag):
  return 't_'+base64.b64encode(tag.encode("utf-8"))

def GenHistoryKeyName(tags):
  tags.sort()
  return 'h_'+base64.b64encode(" ".join(tags).encode("utf-8"))

def GenQueryRecordsByTagKey(begin, end, tags):
  tags.sort()
  return 'q_'+base64.b64encode(" ".join(tags).encode("utf-8") + 
                               begin.strftime("%Y%m%d%H%M%S") +
                               end.strftime("%Y%m%d%H%M%S"))

def GenTagBookmark(bookmark, direction, pagesize):
  if "next" == direction:
    if bookmark is None:
      return "tbk_"+direction+str(pagesize)+base64.b64encode("FirstBookmark")
    return "tbk_"+direction+str(pagesize)+str(bookmark.key())
  elif "related" == direction:
    bookmark.sort()
    return "tbkr_"+direction+str(pagesize)+"_".join(bookmark)

def GenTagIndexKeyName(word):
  return "ti_"+base64.b64encode(word.encode("utf-8"))

def GetRecord(date, tags):
  item = memcache.get(GenRecordKeyName(date, tags))
  if item is not None:
    return item
  ins = Record.get_by_key_name(GenRecordKeyName(date, tags))
  if ins is not None:
    memcache.set(GenRecordKeyName(date, tags), (True, ins))
    return (True, ins)
  else:
    memcache.set(GenRecordKeyName(date, tags), (False, None))
  return (False, None)

def GetTag(tag):
  item = memcache.get(GenTagKeyName(tag))
  if item is not None:
    return item
  ins = Tag.get_by_key_name(GenTagKeyName(tag))
  if ins is not None:
    memcache.set(GenTagKeyName(tag), (True, ins))
    return (True, ins)
  else:
    memcache.set(GenTagKeyName(tag), (False, None))
  return (False, None)

def GetHistory(tags):
  item = memcache.get(GenHistoryKeyName(tags))
  if item is not None:
    return item
  ins = History.get_by_key_name(GenHistoryKeyName(tags))
  if ins is not None:
    memcache.set(GenHistoryKeyName(tags), (True, ins))
    return (True, ins)
  else:
    memcache.set(GenHistoryKeyName(tags), (False, None))
  return (False, None)

def GetTagIndex(word):
  item = memcache.get(GenTagIndexKeyName(word))
  if item is not None:
    return item
  ins = TagIndex.get_by_key_name(GenTagIndexKeyName(word))
  if ins is not None:
    memcache.set(GenTagIndexKeyName(word), (True, ins))
    return (True, ins)
  else:
    memcache.set(GenTagIndexKeyName(word), (False, None))
  return (False, None)

def UpdateEntity(entities, exist):
  e = dict((k, (exist, v)) for k, v in entities.iteritems()) 
  memcache.set_multi(e)

def DBBatch(op, entities, batch=50):
  start = 0
  while len(entities[start:batch+start]) > 0:
    op(entities[start:batch+start])
    start += batch

def GetRecords(date, begin, end, bookmark, pagesize=100):
  """
  http://google-appengine.googlegroups.com/web/efficient_paging_using_key_instead_of_a_dedicated_unique_property.txt
  *Original starting query* 
  WHERE x > 0 AND x < 9
  *Bookmarkable starting query*
  WHERE x > 0 AND x < 9 ORDER BY x ASC, __key__ ASC
  *Derived queries for starting from bookmark entity B*
  WHERE x = B.x AND __key__ > B ORDER BY __key__ ASC
  WHERE x > B.x AND x < 9 ORDER BY x ASC, __key__ ASC 
  """

  gqls = ["WHERE "+date+">=:1 AND "+date+"<=:2 ORDER BY "+date+" ASC, __key__ ASC",
          "WHERE "+date+"=:1 AND __key__>=:2 ORDER BY __key__ ASC",
          "WHERE "+date+">:1 AND "+date+"<=:2 ORDER BY "+date+" ASC, __key__ ASC"
          ]
  if bookmark is None:
    query = Record.gql(gqls[0], begin, end)
    records = query.fetch(pagesize+1)
  else:
    bookmarkv = {"inputtime": bookmark.inputtime,
                 "date": bookmark.date}
    query1 = Record.gql(gqls[1], bookmarkv[date], bookmark.key())
    query2 = Record.gql(gqls[2], bookmarkv[date], end)
    records = query1.fetch(pagesize+1)
    records += query2.fetch(pagesize+1-len(records))
  if len(records)<pagesize+1:
    return records, None
  else:
    return records[:pagesize], records[-1]
  
def QueryRecordsByTag(begin, end, tags):
  item = memcache.get(GenQueryRecordsByTagKey(begin, end, tags))
  if item is not None:
    return item

  recKeys = []
  for tag in tags:
    ins = GetTag(tag)[1]
    if ins is None:
      UpdateHistory(tags, False)
      return []
    if 0 == len(recKeys):
      recKeys = ins.records
    else:
      recKeys = list(set(recKeys) - (set(recKeys) - set(ins.records)))
        
  records = Record.get(recKeys)
  i = 0
  while i<len(records):
    if records[i].date<begin \
    or records[i].date>end:
      del records[i]
    else: i += 1
      
  if len(records)>0:
    memcache.set(GenQueryRecordsByTagKey(begin, end, tags), records)
  UpdateHistory(tags, len(records)>0)
  return records

def QueryTagIndexByWords(words):
  if 0 == len(words): return []
  tags = []
  for word in words:
    index = GetTagIndex(word)[1]
    if index is None:
      UpdateHistory([word], False)
      return []
    if 0 == len(tags):
      tags = index.tags
    else:
      tags = list(set(tags) - (set(tags) - set(index.tags)))
    
  i = 0
  while i<len(tags):
    if words not in tags[i]:
      del tags[i]
    else: i += 1
    
  UpdateHistory([words], len(tags)>0)
  return tags

def DeleteTagRecords(records):
  tags, deleteTags, updateTags = {}, [], []
  for record in records:
    for tag in record.tags:
      if True == tags.has_key(tag):
        t = tags[tag]
      else:
        t = GetTag(tag)[1]
      if t is None: continue
      if record.key() in t.records:
        t.records.remove(record.key())
      if False == tags.has_key(t.name):
        tags[t.name] = t
  for t in tags.values():
    if 0 == len(t.records):
      deleteTags.append(t)
    else:
      updateTags.append(t)
  DBBatch(db.delete, deleteTags)
  UpdateEntity(dict((GenTagKeyName(k.name), None) for k in deleteTags), False)
  DBBatch(db.put, updateTags)
  UpdateEntity(dict((GenTagKeyName(k.name), k) for k in updateTags), True)
  return len(deleteTags)

def GetNextTags(bookmark, prev, pagesize=10):
  """
  http://google-appengine.googlegroups.com/web/efficient_paging_using_key_instead_of_a_dedicated_unique_property.txt
  *Original starting query*
  ORDER BY x DESC
  *Bookmarkable starting query*
  ORDER BY x DESC, __key__ ASC
  *Derived queries for starting from bookmark entity B*
  WHERE x = B.x AND __key__ > B ORDER BY __key__ ASC
  WHERE x < B.x ORDER BY x DESC, __key__ ASC
  """
  paging = memcache.get(GenTagBookmark(bookmark, "next", pagesize))
  if paging is not None:
    tags = paging[0]
    if len(tags)>0 \
    and len(tags) == pagesize:
      return tags, tags[-1], paging[1]
    else:
      return tags, None, paging[1]
      
  gqls = ["ORDER BY count DESC, __key__ ASC",
          "WHERE count = :1 AND __key__>:2 ORDER BY __key__ ASC",
          "WHERE count < :1 ORDER BY count DESC, __key__ ASC",
          ]
  if bookmark is None:
    query = Tag.gql(gqls[0])
    tags = query.fetch(pagesize)
  else:
    query1 = Tag.gql(gqls[1], bookmark.count, bookmark.key())
    query2 = Tag.gql(gqls[2], bookmark.count)
    tags = query1.fetch(pagesize)
    tags += query2.fetch(pagesize-len(tags))
  memcache.set(GenTagBookmark(bookmark, "next", pagesize), (tags, prev))
  if len(tags)<pagesize:
    return tags, None, prev
  else:
    return tags, tags[-1], prev
  
def GetRelatedTags(bookmark, pagesize=10):
  tags = bookmark.split(" ")
  related = memcache.get(GenTagBookmark(tags, "related", pagesize))
  if related is not None:
    if len(related)>0:
      return related, related[-1]
    else:
      return related, None
   
  related = []
  for tag in tags:
    ins = GetTag(tag)[1]
    if ins is None:
      UpdateHistory(tags, False)
      return [], None
    if 0 == len(related):
      related = ins.related
    else:
      related = list(set(related) - (set(related) - set(ins.related)))
  memcache.set(GenTagBookmark(tags, "related", pagesize), related)
  return related, None 

def UpdateHistory(tags, found = True):
  history = GetHistory(tags)[1]
  if history is None:
    tags.sort()
    history = History(key_name=GenHistoryKeyName(tags),
                      tags=" ".join(tags))
  else:
    history.qcount += 1
  history.found = found
  history.timestamp = datetime.datetime.now()
  history.put()
  
def BuildTagIndex(tag, tagIndexes):
  for word in tag:
    if tagIndexes.has_key(GenTagIndexKeyName(word)):
      index = tagIndexes[GenTagIndexKeyName(word)]
    else:
      index = GetTagIndex(word)[1]
      if index is None:
        index = TagIndex(key_name=GenTagIndexKeyName(word), word=word)
    if tag not in index.tags:
      index.tags.append(tag)
      if False == tagIndexes.has_key(GenTagIndexKeyName(word)):
        tagIndexes[GenTagIndexKeyName(word)] = index
